Inicio Explorar Ayuda
Registro Iniciar sesión
J1X
/
huginn
Vigilar 2
Destacar 0
Fork 0

Explorar el Código
Merge pull request #857 from cantino/uri_expand

Add a Liquid filter `uri_expand`

Akinori MUSHA %!s(int64=9) %!d(string=hace) años
padre
1e57f3a457 4ec50836c5
commit
350955d12a
Se han modificado 3 ficheros con 119 adiciones y 0 borrados
Mostrar Estadísticas de Diff
  1. 1 0
      CHANGES.md
  2. 50 0
      app/concerns/liquid_interpolatable.rb
  3. 68 0
      spec/concerns/liquid_interpolatable_spec.rb

+ 1 - 0
CHANGES.md
Ver Fichero 

@@ -1,5 +1,6 @@
1 1 
 # Changes
2 2
3 
+* Jun 15, 2015   - Liquid filter `uri_expand` added.
3 4 
 * Jun 12, 2015   - RSSAgent can now accept an array of URLs.
4 5 
 * Jun 8, 2015    - WebsiteAgent includes a `use_namespaces` option to enable XML namespaces.
5 6 
 * May 27, 2015   - Validation warns user if they have not provided a `path` when using JSONPath in WebsiteAgent.

+ 50 - 0
app/concerns/liquid_interpolatable.rb
Ver Fichero 

@@ -132,6 +132,44 @@ module LiquidInterpolatable
132 132 
       nil
133 133 
     end
134 134
135 
+    # Get the destination URL of a given URL by recursively following
136 
+    # redirects, up to 5 times in a row.  If a given string is not a
137 
+    # valid absolute HTTP URL, or any error occurs while following
138 
+    # redirects, the original string is returned.
139 
+    def uri_expand(url, limit = 5)
140 
+      uri = URI(url)
141 
+
142 
+      http = Faraday.new do |builder|
143 
+        builder.adapter :net_http
144 
+        # builder.use FaradayMiddleware::FollowRedirects, limit: limit
145 
+        # ...does not handle non-HTTP URLs.
146 
+      end
147 
+
148 
+      limit.times do
149 
+        begin
150 
+          case uri
151 
+          when URI::HTTP
152 
+            response = http.head(uri)
153 
+            case response.status
154 
+            when 301, 302, 303, 307
155 
+              if location = response['location']
156 
+                uri += location
157 
+                next
158 
+              end
159 
+            end
160 
+          end
161 
+        rescue URI::Error, Faraday::Error, SystemCallError => e
162 
+          logger.error "#{e.class} in #{__method__}(#{url.inspect}) [uri=#{uri.to_s.inspect}]: #{e.message}:\n#{e.backtrace.join("\n")}"
163 
+        end
164 
+
165 
+        return uri.to_s
166 
+      end
167 
+
168 
+      logger.error "Too many rediretions in #{__method__}(#{url.inspect}) [uri=#{uri.to_s.inspect}]"
169 
+
170 
+      url
171 
+    end
172 
+
135 173 
     # Escape a string for use in XPath expression
136 174 
     def to_xpath(string)
137 175 
       subs = string.to_s.scan(/\G(?:\A\z|[^"]+|[^']+)/).map { |x|
 
@@ -148,6 +186,18 @@ module LiquidInterpolatable
148 186 
         'concat(' << subs.join(', ') << ')'
149 187 
       end
150 188 
     end
189 
+
190 
+    private
191 
+
192 
+    def logger
193 
+      @@logger ||=
194 
+        if defined?(Rails)
195 
+          Rails.logger
196 
+        else
197 
+          require 'logger'
198 
+          Logger.new(STDERR)
199 
+        end
200 
+    end
151 201 
   end
152 202 
   Liquid::Template.register_filter(LiquidInterpolatable::Filters)
153 203

+ 68 - 0
spec/concerns/liquid_interpolatable_spec.rb
Ver Fichero 

@@ -96,4 +96,72 @@ describe LiquidInterpolatable::Filters do
96 96 
       expect(@agent.interpolated['foo']).to eq('/dir/foo/index.html')
97 97 
     end
98 98 
   end
99 
+
100 
+  describe 'uri_expand' do
101 
+    before do
102 
+      stub_request(:head, 'https://t.co.x/aaaa').
103 
+        to_return(status: 301, headers: { Location: 'https://bit.ly.x/bbbb' })
104 
+      stub_request(:head, 'https://bit.ly.x/bbbb').
105 
+        to_return(status: 301, headers: { Location: 'http://tinyurl.com.x/cccc' })
106 
+      stub_request(:head, 'http://tinyurl.com.x/cccc').
107 
+        to_return(status: 301, headers: { Location: 'http://www.example.com/welcome' })
108 
+
109 
+      (1..5).each do |i|
110 
+        stub_request(:head, "http://2many.x/#{i}").
111 
+          to_return(status: 301, headers: { Location: "http://2many.x/#{i+1}" })
112 
+      end
113 
+      stub_request(:head, 'http://2many.x/6').
114 
+        to_return(status: 301, headers: { 'Content-Length' => '5' })
115 
+    end
116 
+
117 
+    it 'should follow redirects' do
118 
+      expect(@filter.uri_expand('https://t.co.x/aaaa')).to eq('http://www.example.com/welcome')
119 
+    end
120 
+
121 
+    it 'should respect the limit for the number of redirects' do
122 
+      expect(@filter.uri_expand('http://2many.x/1')).to eq('http://2many.x/1')
123 
+      expect(@filter.uri_expand('http://2many.x/1', 6)).to eq('http://2many.x/6')
124 
+    end
125 
+
126 
+    it 'should detect a redirect loop' do
127 
+      stub_request(:head, 'http://bad.x/aaaa').
128 
+        to_return(status: 301, headers: { Location: 'http://bad.x/bbbb' })
129 
+      stub_request(:head, 'http://bad.x/bbbb').
130 
+        to_return(status: 301, headers: { Location: 'http://bad.x/aaaa' })
131 
+
132 
+      expect(@filter.uri_expand('http://bad.x/aaaa')).to eq('http://bad.x/aaaa')
133 
+    end
134 
+
135 
+    it 'should be able to handle an FTP URL' do
136 
+      stub_request(:head, 'http://downloads.x/aaaa').
137 
+        to_return(status: 301, headers: { Location: 'http://downloads.x/download?file=aaaa.zip' })
138 
+      stub_request(:head, 'http://downloads.x/download').
139 
+        with(query: { file: 'aaaa.zip' }).
140 
+        to_return(status: 301, headers: { Location: 'ftp://downloads.x/pub/aaaa.zip' })
141 
+
142 
+      expect(@filter.uri_expand('http://downloads.x/aaaa')).to eq('ftp://downloads.x/pub/aaaa.zip')
143 
+    end
144 
+
145 
+    describe 'used in interpolation' do
146 
+      before do
147 
+        @agent = Agents::InterpolatableAgent.new(name: "test")
148 
+      end
149 
+
150 
+      it 'should follow redirects' do
151 
+        @agent.interpolation_context['short_url'] = 'https://t.co.x/aaaa'
152 
+        @agent.options['long_url'] = '{{ short_url | uri_expand }}'
153 
+        expect(@agent.interpolated['long_url']).to eq('http://www.example.com/welcome')
154 
+      end
155 
+
156 
+      it 'should respect the limit for the number of redirects' do
157 
+        @agent.interpolation_context['short_url'] = 'http://2many.x/1'
158 
+        @agent.options['long_url'] = '{{ short_url | uri_expand }}'
159 
+        expect(@agent.interpolated['long_url']).to eq('http://2many.x/1')
160 
+
161 
+        @agent.interpolation_context['short_url'] = 'http://2many.x/1'
162 
+        @agent.options['long_url'] = '{{ short_url | uri_expand:6 }}'
163 
+        expect(@agent.interpolated['long_url']).to eq('http://2many.x/6')
164 
+      end
165 
+    end
166 
+  end
99 167 
 end